{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T19:43:11.276446Z",
     "start_time": "2017-10-27T19:43:10.170421Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "\n",
    "import re\n",
    "import collections\n",
    "import itertools\n",
    "import bcolz\n",
    "import pickle\n",
    "sys.path.append('../../lib')\n",
    "sys.path.append('../')\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import gc\n",
    "import random\n",
    "import smart_open\n",
    "import h5py\n",
    "import csv\n",
    "import json\n",
    "import functools\n",
    "import time\n",
    "import string\n",
    "\n",
    "import datetime as dt\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "import global_utils\n",
    "\n",
    "random_state_number = 967898"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T19:43:11.713448Z",
     "start_time": "2017-10-27T19:43:11.277658Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/gpu:0', '/gpu:1']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.python.client import device_lib\n",
    "def get_available_gpus():\n",
    "    local_device_protos = device_lib.list_local_devices()\n",
    "    return [x.name for x in local_device_protos if x.device_type == 'GPU']\n",
    "\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth=True\n",
    "sess = tf.Session(config=config)\n",
    "get_available_gpus()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T19:43:11.797083Z",
     "start_time": "2017-10-27T19:43:11.714560Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using matplotlib backend: TkAgg\n",
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bicepjai/Programs/anaconda3/envs/dsotc-c3/lib/python3.6/site-packages/IPython/core/magics/pylab.py:160: UserWarning: pylab import has clobbered these variables: ['random']\n",
      "`%matplotlib` prevents importing * from pylab and numpy\n",
      "  \"\\n`%matplotlib` prevents importing * from pylab and numpy\"\n"
     ]
    }
   ],
   "source": [
    "%pylab\n",
    "%matplotlib inline\n",
    "%load_ext line_profiler\n",
    "%load_ext memory_profiler\n",
    "%load_ext autoreload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T19:43:11.801552Z",
     "start_time": "2017-10-27T19:43:11.798386Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.options.mode.chained_assignment = None\n",
    "pd.options.display.max_columns = 999\n",
    "color = sns.color_palette()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T19:43:29.089283Z",
     "start_time": "2017-10-27T19:43:11.802816Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "store = pd.HDFStore('../../data_prep/processed/stage1/data_frames.h5')\n",
    "train_df = store['train_df']\n",
    "test_df = store['test_df']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T19:43:29.410834Z",
     "start_time": "2017-10-27T19:43:29.090475Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Gene</th>\n",
       "      <th>Variation</th>\n",
       "      <th>Class</th>\n",
       "      <th>Sentences</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>[fam58a]</td>\n",
       "      <td>[truncating, mutations]</td>\n",
       "      <td>1</td>\n",
       "      <td>[[cyclin-dependent, kinases, , cdks, , regulat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>[cbl]</td>\n",
       "      <td>[w802*]</td>\n",
       "      <td>2</td>\n",
       "      <td>[[abstract, background, non-small, cell, lung,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>[cbl]</td>\n",
       "      <td>[q249e]</td>\n",
       "      <td>2</td>\n",
       "      <td>[[abstract, background, non-small, cell, lung,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>[cbl]</td>\n",
       "      <td>[n454d]</td>\n",
       "      <td>3</td>\n",
       "      <td>[[recent, evidence, has, demonstrated, that, a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>[cbl]</td>\n",
       "      <td>[l399v]</td>\n",
       "      <td>4</td>\n",
       "      <td>[[oncogenic, mutations, in, the, monomeric, ca...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID      Gene                Variation  Class  \\\n",
       "0   0  [fam58a]  [truncating, mutations]      1   \n",
       "1   1     [cbl]                  [w802*]      2   \n",
       "2   2     [cbl]                  [q249e]      2   \n",
       "3   3     [cbl]                  [n454d]      3   \n",
       "4   4     [cbl]                  [l399v]      4   \n",
       "\n",
       "                                           Sentences  \n",
       "0  [[cyclin-dependent, kinases, , cdks, , regulat...  \n",
       "1  [[abstract, background, non-small, cell, lung,...  \n",
       "2  [[abstract, background, non-small, cell, lung,...  \n",
       "3  [[recent, evidence, has, demonstrated, that, a...  \n",
       "4  [[oncogenic, mutations, in, the, monomeric, ca...  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Gene</th>\n",
       "      <th>Variation</th>\n",
       "      <th>Sentences</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>[acsl4]</td>\n",
       "      <td>[r570s]</td>\n",
       "      <td>[[2, this, mutation, resulted, in, a, myelopro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>[naglu]</td>\n",
       "      <td>[p521l]</td>\n",
       "      <td>[[abstract, the, large, tumor, suppressor, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>[pah]</td>\n",
       "      <td>[l333f]</td>\n",
       "      <td>[[vascular, endothelial, growth, factor, recep...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>[ing1]</td>\n",
       "      <td>[a148d]</td>\n",
       "      <td>[[inflammatory, myofibroblastic, tumor, , imt,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>[tmem216]</td>\n",
       "      <td>[g77a]</td>\n",
       "      <td>[[abstract, retinoblastoma, is, a, pediatric, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID       Gene Variation                                          Sentences\n",
       "0   0    [acsl4]   [r570s]  [[2, this, mutation, resulted, in, a, myelopro...\n",
       "1   1    [naglu]   [p521l]  [[abstract, the, large, tumor, suppressor, 1, ...\n",
       "2   2      [pah]   [l333f]  [[vascular, endothelial, growth, factor, recep...\n",
       "3   3     [ing1]   [a148d]  [[inflammatory, myofibroblastic, tumor, , imt,...\n",
       "4   4  [tmem216]    [g77a]  [[abstract, retinoblastoma, is, a, pediatric, ..."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(train_df.head())\n",
    "display(test_df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T19:43:29.522905Z",
     "start_time": "2017-10-27T19:43:29.411928Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "352220 352220\n"
     ]
    }
   ],
   "source": [
    "corpus_vocab_list, corpus_vocab_wordidx = None, None\n",
    "with open('../../data_prep/processed/stage1/vocab_words_wordidx.pkl', 'rb') as f:\n",
    "    (corpus_vocab_list, corpus_wordidx) = pickle.load(f)\n",
    "print(len(corpus_vocab_list), len(corpus_wordidx))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-14T08:20:17.449244Z",
     "start_time": "2017-08-14T08:20:15.593136Z"
    },
    "collapsed": true
   },
   "source": [
    "# Data Prep"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To control the vocabulary pass in updated corpus_wordidx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T19:43:29.562987Z",
     "start_time": "2017-10-27T19:43:29.524001Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2988, 5)\n",
      "(333, 5)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "x_train_df, x_val_df = train_test_split(train_df,\n",
    "                                         test_size=0.10, random_state=random_state_number,\n",
    "                                         stratify=train_df.Class)\n",
    "\n",
    "print(x_train_df.shape)\n",
    "print(x_val_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T19:43:36.684316Z",
     "start_time": "2017-10-27T19:43:29.564090Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.contrib.keras.python.keras.utils import np_utils\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils.np_utils import to_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T19:43:36.688821Z",
     "start_time": "2017-10-27T19:43:36.685967Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vocab_size=len(corpus_vocab_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## T:sent_words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### generate data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T19:43:36.707838Z",
     "start_time": "2017-10-27T19:43:36.690359Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "custom_unit_dict = {\n",
    "         \"gene_unit\"      : \"words\",\n",
    "         \"variation_unit\" : \"words\",\n",
    "         # text transformed to sentences attribute\n",
    "         \"doc_unit\"       : \"words\",\n",
    "         \"doc_form\"       : \"sentences\",\n",
    "         \"divide_document\": \"multiple_unit\"\n",
    "      }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T19:44:41.443781Z",
     "start_time": "2017-10-27T19:43:36.709139Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%autoreload\n",
    "import global_utils\n",
    "gen_data = global_utils.GenerateDataset(x_train_df, corpus_wordidx)\n",
    "x_train_21_T, x_train_21_G, x_train_21_V, x_train_21_C = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                             has_class=True,\n",
    "                                                                             add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T19:44:43.261910Z",
     "start_time": "2017-10-27T19:44:41.444973Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train data\n",
      "(1086419,) [352216, 252037, 202038, 70974, 86431, 164788, 109857, 338562, 123191, 209585, 221967, 49123, 331220, 140212, 209585, 229015, 140770, 182848, 111721, 8208, 0, 352217]\n",
      "(1086419, 3) [352216, 164788, 352217]\n",
      "(1086419,) [352216, 86196, 352217]\n",
      "(1086419,) 4\n"
     ]
    }
   ],
   "source": [
    "print(\"Train data\")\n",
    "print(np.array(x_train_21_T).shape, x_train_21_T[0])\n",
    "print(np.array(x_train_21_G).shape, x_train_21_G[0])\n",
    "print(np.array(x_train_21_V).shape, x_train_21_V[0])\n",
    "print(np.array(x_train_21_C).shape, x_train_21_C[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T19:44:49.845159Z",
     "start_time": "2017-10-27T19:44:43.263112Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gen_data = global_utils.GenerateDataset(x_val_df, corpus_wordidx)\n",
    "x_val_21_T, x_val_21_G, x_val_21_V, x_val_21_C = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                             has_class=True,\n",
    "                                                                             add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T19:44:50.067143Z",
     "start_time": "2017-10-27T19:44:49.846356Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val data\n",
      "text (128341,)\n",
      "gene (128341, 3) [352216, 217983, 352217]\n",
      "variation (128341,) [352216, 41934, 352217]\n",
      "classes (128341,) 4\n"
     ]
    }
   ],
   "source": [
    "print(\"Val data\")\n",
    "print(\"text\",np.array(x_val_21_T).shape)\n",
    "print(\"gene\",np.array(x_val_21_G).shape, x_val_21_G[0])\n",
    "print(\"variation\",np.array(x_val_21_V).shape, x_val_21_V[0])\n",
    "print(\"classes\",np.array(x_val_21_C).shape, x_val_21_C[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### format data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T19:44:50.071183Z",
     "start_time": "2017-10-27T19:44:50.068584Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "word_unknown_tag_idx   = corpus_wordidx[\"<UNK>\"]\n",
    "char_unknown_tag_idx   = global_utils.char_unknown_tag_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T19:44:50.103965Z",
     "start_time": "2017-10-27T19:44:50.072425Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "MAX_SENT_LEN = 60"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T19:44:56.436234Z",
     "start_time": "2017-10-27T19:44:50.105090Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1086419, 60) (128341, 60)\n"
     ]
    }
   ],
   "source": [
    "x_train_21_T = pad_sequences(x_train_21_T, maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx,\n",
    "                                  padding=\"post\",truncating=\"post\")\n",
    "x_val_21_T = pad_sequences(x_val_21_T, maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx,\n",
    "                                  padding=\"post\",truncating=\"post\")\n",
    "print(x_train_21_T.shape, x_val_21_T.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "keras np_utils.to_categorical expects zero index categorical variables\n",
    "\n",
    "https://github.com/fchollet/keras/issues/570"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T19:44:56.535348Z",
     "start_time": "2017-10-27T19:44:56.437453Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train_21_C = np.array(x_train_21_C) - 1\n",
    "x_val_21_C = np.array(x_val_21_C) - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T19:44:56.568402Z",
     "start_time": "2017-10-27T19:44:56.536779Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1086419, 9) (128341, 9)\n"
     ]
    }
   ],
   "source": [
    "x_train_21_C = np_utils.to_categorical(np.array(x_train_21_C), 9)\n",
    "x_val_21_C = np_utils.to_categorical(np.array(x_val_21_C), 9)\n",
    "print(x_train_21_C.shape, x_val_21_C.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## T:text_words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### generate data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T02:46:54.083902Z",
     "start_time": "2017-10-27T02:46:54.078362Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "custom_unit_dict = {\n",
    "         \"gene_unit\"      : \"words\",\n",
    "         \"variation_unit\" : \"words\",\n",
    "         # text transformed to sentences attribute\n",
    "         \"doc_unit\"       : \"words\",\n",
    "         \"doc_form\"       : \"text\",\n",
    "         \"divide_document\": \"single_unit\"\n",
    "      }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T02:47:05.636131Z",
     "start_time": "2017-10-27T02:46:55.087119Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "%autoreload\n",
    "import global_utils\n",
    "gen_data = global_utils.GenerateDataset(x_train_df, corpus_wordidx)\n",
    "x_train_22_T, x_train_22_G, x_train_22_V, x_train_22_C = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                             has_class=True,\n",
    "                                                                             add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T02:47:06.673362Z",
     "start_time": "2017-10-27T02:47:05.637277Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train data\n",
      "text (2988,)\n",
      "gene (2988, 3) [352216, 164788, 352217]\n",
      "variation (2988,) [352216, 86196, 352217]\n",
      "classes (2988,) 4\n"
     ]
    }
   ],
   "source": [
    "print(\"Train data\")\n",
    "print(\"text\",np.array(x_train_22_T).shape)\n",
    "print(\"gene\",np.array(x_train_22_G).shape, x_train_22_G[0])\n",
    "print(\"variation\",np.array(x_train_22_V).shape, x_train_22_V[0])\n",
    "print(\"classes\",np.array(x_train_22_C).shape, x_train_22_C[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T02:47:07.863793Z",
     "start_time": "2017-10-27T02:47:06.675212Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "gen_data = global_utils.GenerateDataset(x_val_df, corpus_wordidx)\n",
    "x_val_22_T, x_val_22_G, x_val_22_V, x_val_22_C = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                             has_class=True,\n",
    "                                                                             add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T02:47:07.991209Z",
     "start_time": "2017-10-27T02:47:07.865046Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val data\n",
      "text (333,)\n",
      "gene (333, 3) [352216, 217983, 352217]\n",
      "variation (333,) [352216, 41934, 352217]\n",
      "classes (333,) 4\n"
     ]
    }
   ],
   "source": [
    "print(\"Val data\")\n",
    "print(\"text\",np.array(x_val_22_T).shape)\n",
    "print(\"gene\",np.array(x_val_22_G).shape, x_val_22_G[0])\n",
    "print(\"variation\",np.array(x_val_22_V).shape, x_val_22_V[0])\n",
    "print(\"classes\",np.array(x_val_22_C).shape, x_val_22_C[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### format data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T02:47:08.015565Z",
     "start_time": "2017-10-27T02:47:07.992509Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "word_unknown_tag_idx   = corpus_wordidx[\"<UNK>\"]\n",
    "char_unknown_tag_idx   = global_utils.char_unknown_tag_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T02:47:08.036767Z",
     "start_time": "2017-10-27T02:47:08.016770Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "MAX_TEXT_LEN = 5000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T02:47:09.113995Z",
     "start_time": "2017-10-27T02:47:08.037908Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2988, 5000) (333, 5000)\n"
     ]
    }
   ],
   "source": [
    "x_train_22_T = pad_sequences(x_train_22_T, maxlen=MAX_TEXT_LEN, value=word_unknown_tag_idx,\n",
    "                                  padding=\"post\",truncating=\"post\")\n",
    "x_val_22_T = pad_sequences(x_val_22_T, maxlen=MAX_TEXT_LEN, value=word_unknown_tag_idx,\n",
    "                                  padding=\"post\",truncating=\"post\")\n",
    "print(x_train_22_T.shape, x_val_22_T.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T02:47:09.151860Z",
     "start_time": "2017-10-27T02:47:09.115126Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2988, 1) (2988, 4)\n",
      "(333, 1) (333, 4)\n"
     ]
    }
   ],
   "source": [
    "MAX_GENE_LEN = 1\n",
    "MAX_VAR_LEN = 4\n",
    "x_train_22_G = pad_sequences(x_train_22_G, maxlen=MAX_GENE_LEN, value=word_unknown_tag_idx)\n",
    "x_train_22_V = pad_sequences(x_train_22_V, maxlen=MAX_VAR_LEN, value=word_unknown_tag_idx)\n",
    "\n",
    "x_val_22_G = pad_sequences(x_val_22_G, maxlen=MAX_GENE_LEN, value=word_unknown_tag_idx)\n",
    "x_val_22_V = pad_sequences(x_val_22_V, maxlen=MAX_VAR_LEN, value=word_unknown_tag_idx)\n",
    "\n",
    "print(x_train_22_G.shape, x_train_22_V.shape)\n",
    "print(x_val_22_G.shape, x_val_22_V.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "keras np_utils.to_categorical expects zero index categorical variables\n",
    "\n",
    "https://github.com/fchollet/keras/issues/570"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T02:47:09.155774Z",
     "start_time": "2017-10-27T02:47:09.152976Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "x_train_22_C = np.array(x_train_22_C) - 1\n",
    "x_val_22_C = np.array(x_val_22_C) - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T02:47:09.188518Z",
     "start_time": "2017-10-27T02:47:09.157136Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2988, 9) (333, 9)\n"
     ]
    }
   ],
   "source": [
    "x_train_22_C = np_utils.to_categorical(np.array(x_train_22_C), 9)\n",
    "x_val_22_C = np_utils.to_categorical(np.array(x_val_22_C), 9)\n",
    "print(x_train_22_C.shape, x_val_22_C.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### test Data setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:43:32.887420Z",
     "start_time": "2017-09-26T03:43:29.372697Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "gen_data = global_utils.GenerateDataset(test_df, corpus_wordidx)\n",
    "x_test_22_T, x_test_22_G, x_test_22_V, _ = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                has_class=False,\n",
    "                                                                add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:43:33.178763Z",
     "start_time": "2017-09-26T03:43:32.888877Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test data\n",
      "text (986,)\n",
      "gene (986, 3) [364606, 188717, 364607]\n",
      "variation (986,) [364606, 317947, 364607]\n"
     ]
    }
   ],
   "source": [
    "print(\"Test data\")\n",
    "print(\"text\",np.array(x_test_22_T).shape)\n",
    "print(\"gene\",np.array(x_test_22_G).shape, x_test_22_G[0])\n",
    "print(\"variation\",np.array(x_test_22_V).shape, x_test_22_V[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:43:33.546461Z",
     "start_time": "2017-09-26T03:43:33.181689Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(986, 5000)\n"
     ]
    }
   ],
   "source": [
    "x_test_22_T = pad_sequences(x_test_22_T, maxlen=MAX_TEXT_LEN, value=word_unknown_tag_idx,\n",
    "                                  padding=\"post\",truncating=\"post\")\n",
    "print(x_test_22_T.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:43:33.575305Z",
     "start_time": "2017-09-26T03:43:33.548386Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(986, 1) (986, 4)\n"
     ]
    }
   ],
   "source": [
    "MAX_GENE_LEN = 1\n",
    "MAX_VAR_LEN = 4\n",
    "x_test_22_G = pad_sequences(x_test_22_G, maxlen=MAX_GENE_LEN, value=word_unknown_tag_idx)\n",
    "x_test_22_V = pad_sequences(x_test_22_V, maxlen=MAX_VAR_LEN, value=word_unknown_tag_idx)\n",
    "\n",
    "print(x_test_22_G.shape, x_test_22_V.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## T:text_chars"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### generate data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:07:49.588270Z",
     "start_time": "2017-10-26T23:07:49.582681Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "custom_unit_dict = {\n",
    "         \"gene_unit\"          : \"raw_chars\",\n",
    "         \"variation_unit\"     : \"raw_chars\",\n",
    "         # text transformed to sentences attribute\n",
    "         \"doc_unit\"           : \"raw_chars\",\n",
    "         \"doc_form\"           : \"text\",\n",
    "         \"divide_document\"    : \"multiple_unit\"\n",
    "      }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:09:42.095904Z",
     "start_time": "2017-10-26T23:07:50.005896Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "%autoreload\n",
    "import global_utils\n",
    "gen_data = global_utils.GenerateDataset(x_train_df, corpus_wordidx)\n",
    "x_train_33_T, x_train_33_G, x_train_33_V, x_train_33_C = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                             has_class=True,\n",
    "                                                                             add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:09:48.474776Z",
     "start_time": "2017-10-26T23:09:42.097144Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train data\n",
      "text (1086419,) [74, 71, 19, 7, 4, 72, 71, 19, 20, 12, 14, 17, 72, 71, 18, 20, 15, 15, 17, 4, 18, 18, 14, 17, 72, 71, 6, 4, 13, 4, 72, 71, 15, 19, 4, 13, 72, 71, 8, 18, 72, 71, 5, 17, 4, 16, 20, 4, 13, 19, 11, 24, 72, 71, 12, 20, 19, 0, 19, 4, 3, 72, 71, 8, 13, 72, 71, 3, 8, 21, 4, 17, 18, 4, 72, 71, 7, 20, 12, 0, 13, 72, 71, 2, 0, 13, 2, 4, 17, 18, 72, 71, 0, 13, 3, 72, 71, 8, 13, 72, 71, 0, 20, 19, 14, 18, 14, 12, 0, 11, 72, 71, 3, 14, 12, 8, 13, 0, 13, 19, 72, 71, 2, 0, 13, 2, 4, 17, 72, 71, 15, 17, 4, 3, 8, 18, 15, 14, 18, 8, 19, 8, 14, 13, 72, 71, 3, 8, 18, 14, 17, 3, 4, 17, 18, 72, 71, 72, 75]\n",
      "gene (1086419,) [74, 71, 15, 19, 4, 13, 72, 75]\n",
      "variation (1086419,) [74, 71, 24, 27, 32, 2, 72, 75]\n",
      "classes (1086419,) 4\n"
     ]
    }
   ],
   "source": [
    "print(\"Train data\")\n",
    "print(\"text\",np.array(x_train_33_T).shape, x_train_33_T[0])\n",
    "print(\"gene\",np.array(x_train_33_G).shape, x_train_33_G[0])\n",
    "print(\"variation\",np.array(x_train_33_V).shape, x_train_33_V[0])\n",
    "print(\"classes\",np.array(x_train_33_C).shape, x_train_33_C[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:10:05.745670Z",
     "start_time": "2017-10-26T23:09:48.475925Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "%autoreload\n",
    "import global_utils\n",
    "gen_data = global_utils.GenerateDataset(x_val_df, corpus_wordidx)\n",
    "x_val_33_T, x_val_33_G, x_val_33_V, x_val_33_C = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                             has_class=True,\n",
    "                                                                             add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:10:06.510202Z",
     "start_time": "2017-10-26T23:10:05.746898Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val data\n",
      "text (128341,) [74, 71, 0, 19, 72, 71, 19, 7, 8, 18, 72, 71, 19, 8, 12, 4, 72, 71, 15, 14, 8, 13, 19, 72, 71, 72, 71, 19, 7, 4, 72, 71, 4, 23, 15, 17, 4, 18, 18, 8, 14, 13, 72, 71, 14, 5, 72, 71, 22, 8, 11, 3, 36, 19, 24, 15, 4, 72, 71, 15, 27, 32, 8, 13, 10, 30, 0, 72, 71, 8, 13, 72, 71, 20, 28, 14, 18, 72, 71, 2, 4, 11, 11, 18, 72, 71, 8, 13, 3, 20, 2, 4, 3, 72, 71, 15, 14, 19, 4, 13, 19, 72, 71, 2, 4, 11, 11, 72, 71, 2, 24, 2, 11, 4, 72, 71, 0, 17, 17, 4, 18, 19, 72, 71, 0, 19, 72, 71, 1, 14, 19, 7, 72, 71, 19, 4, 12, 15, 4, 17, 0, 19, 20, 17, 4, 18, 72, 71, 72, 71, 15, 27, 32, 8, 13, 10, 30, 0, 72, 71, 8, 13, 3, 20, 2, 4, 3, 72, 71, 18, 36, 15, 7, 0, 18, 4, 72, 71, 8, 13, 7, 8, 1, 8, 19, 8, 14, 13, 72, 71, 14, 5, 72, 71, 30, 28, 33, 29, 72, 71, 72, 71, 0, 13, 3, 72, 71, 30, 35, 33, 29, 72, 71, 72, 71, 0, 19, 72, 71, 29, 33, 27, 2, 72, 71, 0, 13, 3, 72, 71, 30, 26, 27, 2, 72, 71, 72, 71, 17, 4, 18, 15, 4, 2, 19, 8, 21, 4, 11, 24, 72, 71, 72, 75]\n",
      "gene (128341,) [74, 71, 2, 3, 10, 13, 28, 0, 72, 75]\n",
      "variation (128341,) [74, 71, 0, 32, 26, 21, 72, 75]\n",
      "classes (128341,) 4\n"
     ]
    }
   ],
   "source": [
    "print(\"Val data\")\n",
    "print(\"text\",np.array(x_val_33_T).shape, x_val_33_T[98])\n",
    "print(\"gene\",np.array(x_val_33_G).shape, x_val_33_G[0])\n",
    "print(\"variation\",np.array(x_val_33_V).shape, x_val_33_V[0])\n",
    "print(\"classes\",np.array(x_val_33_C).shape, x_val_33_C[0])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### format data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:10:06.513430Z",
     "start_time": "2017-10-26T23:10:06.511325Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "word_unknown_tag_idx   = corpus_wordidx[\"<UNK>\"]\n",
    "char_unknown_tag_idx   = global_utils.char_unknown_tag_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:10:06.527659Z",
     "start_time": "2017-10-26T23:10:06.514422Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "MAX_CHAR_IN_SENT_LEN = 150"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:10:18.903599Z",
     "start_time": "2017-10-26T23:10:06.529246Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1086419, 150) (128341, 150)\n"
     ]
    }
   ],
   "source": [
    "x_train_33_T = pad_sequences(x_train_33_T, maxlen=MAX_CHAR_IN_SENT_LEN, value=char_unknown_tag_idx,\n",
    "                                  padding=\"post\",truncating=\"post\")\n",
    "x_val_33_T = pad_sequences(x_val_33_T, maxlen=MAX_CHAR_IN_SENT_LEN, value=char_unknown_tag_idx,\n",
    "                                  padding=\"post\",truncating=\"post\")\n",
    "print(x_train_33_T.shape, x_val_33_T.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:10:27.876369Z",
     "start_time": "2017-10-26T23:10:18.905017Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1086419, 150) (1086419, 150)\n",
      "(128341, 150) (128341, 150)\n"
     ]
    }
   ],
   "source": [
    "x_train_33_G = pad_sequences(x_train_33_G, maxlen=MAX_CHAR_IN_SENT_LEN, value=char_unknown_tag_idx)\n",
    "x_train_33_V = pad_sequences(x_train_33_V, maxlen=MAX_CHAR_IN_SENT_LEN, value=char_unknown_tag_idx)\n",
    "\n",
    "x_val_33_G = pad_sequences(x_val_33_G, maxlen=MAX_CHAR_IN_SENT_LEN, value=char_unknown_tag_idx)\n",
    "x_val_33_V = pad_sequences(x_val_33_V, maxlen=MAX_CHAR_IN_SENT_LEN, value=char_unknown_tag_idx)\n",
    "\n",
    "print(x_train_33_G.shape, x_train_33_V.shape)\n",
    "print(x_val_33_G.shape, x_val_33_V.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "keras np_utils.to_categorical expects zero index categorical variables\n",
    "\n",
    "https://github.com/fchollet/keras/issues/570"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:10:27.985411Z",
     "start_time": "2017-10-26T23:10:27.877544Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "x_train_33_C = np.array(x_train_33_C) - 1\n",
    "x_val_33_C = np.array(x_val_33_C) - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:10:28.016098Z",
     "start_time": "2017-10-26T23:10:27.986732Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1086419, 9) (128341, 9)\n"
     ]
    }
   ],
   "source": [
    "x_train_33_C = np_utils.to_categorical(np.array(x_train_33_C), 9)\n",
    "x_val_33_C = np_utils.to_categorical(np.array(x_val_33_C), 9)\n",
    "print(x_train_33_C.shape, x_val_33_C.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## T:text_sent_words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### generate data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T02:20:00.685040Z",
     "start_time": "2017-10-27T02:20:00.670683Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "custom_unit_dict = {\n",
    "         \"gene_unit\"          : \"words\",\n",
    "         \"variation_unit\"     : \"words\",\n",
    "         # text transformed to sentences attribute\n",
    "         \"doc_unit\"           : \"word_list\",\n",
    "         \"doc_form\"           : \"text\",\n",
    "         \"divide_document\"    : \"single_unit\"\n",
    "      }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T02:20:15.952551Z",
     "start_time": "2017-10-27T02:20:00.686423Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "%autoreload\n",
    "import global_utils\n",
    "gen_data = global_utils.GenerateDataset(x_train_df, corpus_wordidx)\n",
    "x_train_34_T, x_train_34_G, x_train_34_V, x_train_34_C = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                             has_class=True,\n",
    "                                                                             add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T02:20:17.054217Z",
     "start_time": "2017-10-27T02:20:15.953722Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train data\n",
      "text (2988,) [[352216, 252037, 202038, 70974, 86431, 164788, 109857, 338562, 123191, 209585, 221967, 49123, 331220, 140212, 209585, 229015, 140770, 182848, 111721, 8208, 0, 352217]]\n",
      "gene (2988, 3) [352216, 164788, 352217]\n",
      "variation (2988,) [352216, 86196, 352217]\n",
      "classes (2988,) 4\n"
     ]
    }
   ],
   "source": [
    "print(\"Train data\")\n",
    "print(\"text\",np.array(x_train_34_T).shape, x_train_34_T[0][:1])\n",
    "print(\"gene\",np.array(x_train_34_G).shape, x_train_34_G[0])\n",
    "print(\"variation\",np.array(x_train_34_V).shape, x_train_34_V[0])\n",
    "print(\"classes\",np.array(x_train_34_C).shape, x_train_34_C[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T02:20:18.329383Z",
     "start_time": "2017-10-27T02:20:17.055319Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "%autoreload\n",
    "import global_utils\n",
    "gen_data = global_utils.GenerateDataset(x_val_df, corpus_wordidx)\n",
    "x_val_34_T, x_val_34_G, x_val_34_V, x_val_34_C = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                             has_class=True,\n",
    "                                                                             add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T02:20:18.467889Z",
     "start_time": "2017-10-27T02:20:18.330665Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val data\n",
      "text (333,) [[352216, 252037, 156537, 91785, 67201, 109857, 123191, 209585, 213751, 5638, 0, 126280, 49123, 331220, 0, 352217]]\n",
      "gene (333, 3) [352216, 217983, 352217]\n",
      "variation (333,) [352216, 41934, 352217]\n",
      "classes (333,) 4\n"
     ]
    }
   ],
   "source": [
    "print(\"Val data\")\n",
    "print(\"text\",np.array(x_val_34_T).shape, x_val_34_T[98][:1])\n",
    "print(\"gene\",np.array(x_val_34_G).shape, x_val_34_G[0])\n",
    "print(\"variation\",np.array(x_val_34_V).shape, x_val_34_V[0])\n",
    "print(\"classes\",np.array(x_val_34_C).shape, x_val_34_C[0])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### format data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T02:20:18.479648Z",
     "start_time": "2017-10-27T02:20:18.468992Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "word_unknown_tag_idx   = corpus_wordidx[\"<UNK>\"]\n",
    "char_unknown_tag_idx   = global_utils.char_unknown_tag_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T02:20:18.502495Z",
     "start_time": "2017-10-27T02:20:18.481643Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "MAX_DOC_LEN = 500 # no of sentences in a document\n",
    "MAX_SENT_LEN = 80 # no of words in a sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T02:20:28.645457Z",
     "start_time": "2017-10-27T02:20:18.504233Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "for doc_i, doc in enumerate(x_train_34_T):\n",
    "    x_train_34_T[doc_i] = x_train_34_T[doc_i][:MAX_DOC_LEN]\n",
    "    # padding sentences\n",
    "    if len(x_train_34_T[doc_i]) < MAX_DOC_LEN:\n",
    "        for not_used_i in range(0,MAX_DOC_LEN - len(x_train_34_T[doc_i])):\n",
    "            x_train_34_T[doc_i].append([word_unknown_tag_idx]*MAX_SENT_LEN)\n",
    "    # padding words\n",
    "    x_train_34_T[doc_i] = pad_sequences(x_train_34_T[doc_i], maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx)\n",
    "    \n",
    "for doc_i, doc in enumerate(x_val_34_T):\n",
    "    x_val_34_T[doc_i] = x_val_34_T[doc_i][:MAX_DOC_LEN]\n",
    "    # padding sentences\n",
    "    if len(x_val_34_T[doc_i]) < MAX_DOC_LEN:\n",
    "        for not_used_i in range(0,MAX_DOC_LEN - len(x_val_34_T[doc_i])):\n",
    "            x_val_34_T[doc_i].append([word_unknown_tag_idx]*MAX_SENT_LEN)\n",
    "    # padding words\n",
    "    x_val_34_T[doc_i] = pad_sequences(x_val_34_T[doc_i], maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx)\n",
    "    \n",
    "x_train_34_T = np.array(x_train_34_T)\n",
    "x_val_34_T = np.array(x_val_34_T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T02:20:28.650166Z",
     "start_time": "2017-10-27T02:20:28.646889Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(333, 500, 80) (2988, 500, 80)\n"
     ]
    }
   ],
   "source": [
    "print(x_val_34_T.shape, x_train_34_T.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T02:20:28.722888Z",
     "start_time": "2017-10-27T02:20:28.651790Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2988, 80) (2988, 80)\n",
      "(333, 80) (333, 80)\n"
     ]
    }
   ],
   "source": [
    "x_train_34_G = pad_sequences(x_train_34_G, maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx)\n",
    "x_train_34_V = pad_sequences(x_train_34_V, maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx)\n",
    "\n",
    "x_val_34_G = pad_sequences(x_val_34_G, maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx)\n",
    "x_val_34_V = pad_sequences(x_val_34_V, maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx)\n",
    "\n",
    "print(x_train_34_G.shape, x_train_34_V.shape)\n",
    "print(x_val_34_G.shape, x_val_34_V.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "keras np_utils.to_categorical expects zero index categorical variables\n",
    "\n",
    "https://github.com/fchollet/keras/issues/570"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T02:20:28.728851Z",
     "start_time": "2017-10-27T02:20:28.724555Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "x_train_34_C = np.array(x_train_34_C) - 1\n",
    "x_val_34_C = np.array(x_val_34_C) - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T02:20:28.742924Z",
     "start_time": "2017-10-27T02:20:28.730480Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2988, 9) (333, 9)\n"
     ]
    }
   ],
   "source": [
    "x_train_34_C = np_utils.to_categorical(np.array(x_train_34_C), 9)\n",
    "x_val_34_C = np_utils.to_categorical(np.array(x_val_34_C), 9)\n",
    "print(x_train_34_C.shape, x_val_34_C.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "source": [
    "Need to form 3 dimensional target data for rationale model training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T02:20:28.795678Z",
     "start_time": "2017-10-27T02:20:28.744106Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2988, 500, 9) (333, 500, 9)\n"
     ]
    }
   ],
   "source": [
    "temp = (x_train_34_C.shape[0],1,x_train_34_C.shape[1])\n",
    "x_train_34_C_sent = np.repeat(x_train_34_C.reshape(temp[0],temp[1],temp[2]), MAX_DOC_LEN, axis=1)\n",
    "\n",
    "#sentence test targets\n",
    "temp = (x_val_34_C.shape[0],1,x_val_34_C.shape[1])\n",
    "x_val_34_C_sent = np.repeat(x_val_34_C.reshape(temp[0],temp[1],temp[2]), MAX_DOC_LEN, axis=1)\n",
    "\n",
    "print(x_train_34_C_sent.shape, x_val_34_C_sent.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Embedding layer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### for words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T19:44:56.572011Z",
     "start_time": "2017-10-27T19:44:56.569590Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "WORD_EMB_SIZE = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T19:45:20.077444Z",
     "start_time": "2017-10-27T19:44:56.573346Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(352220, 200)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%autoreload\n",
    "import global_utils\n",
    "ft_file_path = \"/home/bicepjai/Projects/Deep-Survey-Text-Classification/data_prep/processed/stage1/pretrained_word_vectors/ft_sg_200d_50e.vec\"\n",
    "trained_embeddings = global_utils.get_embeddings_from_ft(ft_file_path, WORD_EMB_SIZE, corpus_vocab_list)\n",
    "trained_embeddings.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### for characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:10:28.019172Z",
     "start_time": "2017-10-26T23:10:28.017168Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "CHAR_EMB_SIZE = 64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T23:10:28.051353Z",
     "start_time": "2017-10-26T23:10:28.020527Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(75, 64)"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "char_embeddings = np.random.randn(global_utils.CHAR_ALPHABETS_LEN, CHAR_EMB_SIZE)\n",
    "char_embeddings.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## prep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T19:45:45.626100Z",
     "start_time": "2017-10-27T19:45:45.564438Z"
    }
   },
   "outputs": [],
   "source": [
    "%autoreload\n",
    "import tensorflow.contrib.keras as keras\n",
    "import tensorflow as tf\n",
    "\n",
    "from keras import backend as K\n",
    "\n",
    "from keras.engine import Layer, InputSpec, InputLayer\n",
    "\n",
    "from keras.models import Model, Sequential\n",
    "\n",
    "from keras.layers import Dropout, Embedding, concatenate\n",
    "from keras.layers import Conv1D, MaxPool1D, Conv2D, MaxPool2D, ZeroPadding1D, GlobalMaxPool1D\n",
    "from keras.layers import Dense, Input, Flatten, BatchNormalization\n",
    "from keras.layers import Concatenate, Dot, Merge, Multiply, RepeatVector\n",
    "from keras.layers import Bidirectional, TimeDistributed\n",
    "from keras.layers import SimpleRNN, LSTM, GRU, Lambda, Permute\n",
    "\n",
    "from keras.layers.core import Reshape, Activation\n",
    "from keras.optimizers import Adam\n",
    "from keras.callbacks import ModelCheckpoint,EarlyStopping,TensorBoard\n",
    "from keras.constraints import maxnorm\n",
    "from keras.regularizers import l2\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-24T06:58:17.661183Z",
     "start_time": "2017-08-24T06:58:17.655020Z"
    }
   },
   "source": [
    "## model_1: paper"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Discriminative model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T18:11:44.647758Z",
     "start_time": "2017-10-27T18:11:43.941642Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "text_seq_input = Input(shape=(MAX_SENT_LEN,), dtype='int32')\n",
    "text_embedding = Embedding(vocab_size, WORD_EMB_SIZE, input_length=MAX_SENT_LEN,\n",
    "                            weights=[trained_embeddings], trainable=True)(text_seq_input)\n",
    "\n",
    "dis_model_1 = Sequential([\n",
    "    Embedding(vocab_size, WORD_EMB_SIZE, weights=[trained_embeddings], \n",
    "               input_length=MAX_SENT_LEN, trainable=True),\n",
    "    LSTM(32, return_sequences=True),\n",
    "    Flatten(),\n",
    "    Dense(9, activation='softmax')\n",
    "])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T18:11:47.084682Z",
     "start_time": "2017-10-27T18:11:47.029630Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_11 (Embedding)     (None, 60, 200)           70444000  \n",
      "_________________________________________________________________\n",
      "lstm_5 (LSTM)                (None, 60, 32)            29824     \n",
      "_________________________________________________________________\n",
      "flatten_2 (Flatten)          (None, 1920)              0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 9)                 17289     \n",
      "=================================================================\n",
      "Total params: 70,491,113\n",
      "Trainable params: 70,491,113\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "dis_model_1.compile(loss='categorical_crossentropy', optimizer=Adam(), metrics=['categorical_accuracy'])\n",
    "dis_model_1.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T18:07:53.328650Z",
     "start_time": "2017-10-27T18:07:53.026853Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%rm -rf ./tb_graphs/*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T18:11:53.538422Z",
     "start_time": "2017-10-27T18:11:53.533809Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tb_callback = keras.callbacks.TensorBoard(log_dir='./tb_graphs', histogram_freq=0, write_graph=True, write_images=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T18:11:53.824502Z",
     "start_time": "2017-10-27T18:11:53.819612Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "checkpointer = ModelCheckpoint(filepath=\"dis_model_1_weights.hdf5\", \n",
    "                                    verbose=1,\n",
    "                                    monitor=\"val_categorical_accuracy\",\n",
    "                                    save_best_only=True,\n",
    "                                    mode=\"max\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T18:25:43.377536Z",
     "start_time": "2017-10-27T18:12:01.475318Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no checkpoints available !\n",
      "Train on 1086419 samples, validate on 128341 samples\n",
      "Epoch 1/5\n",
      "1085440/1086419 [============================>.] - ETA: 0s - loss: 1.1275 - categorical_accuracy: 0.5818Epoch 00000: val_categorical_accuracy improved from -inf to 0.57366, saving model to dis_model_weights.hdf5\n",
      "1086419/1086419 [==============================] - 160s - loss: 1.1274 - categorical_accuracy: 0.5818 - val_loss: 1.2273 - val_categorical_accuracy: 0.5737\n",
      "Epoch 2/5\n",
      "1085440/1086419 [============================>.] - ETA: 0s - loss: 0.8648 - categorical_accuracy: 0.6623Epoch 00001: val_categorical_accuracy improved from 0.57366 to 0.58034, saving model to dis_model_weights.hdf5\n",
      "1086419/1086419 [==============================] - 166s - loss: 0.8648 - categorical_accuracy: 0.6623 - val_loss: 1.2828 - val_categorical_accuracy: 0.5803\n",
      "Epoch 3/5\n",
      "1085440/1086419 [============================>.] - ETA: 0s - loss: 0.7795 - categorical_accuracy: 0.6854Epoch 00002: val_categorical_accuracy improved from 0.58034 to 0.58813, saving model to dis_model_weights.hdf5\n",
      "1086419/1086419 [==============================] - 169s - loss: 0.7795 - categorical_accuracy: 0.6854 - val_loss: 1.3735 - val_categorical_accuracy: 0.5881\n",
      "Epoch 4/5\n",
      "1085440/1086419 [============================>.] - ETA: 0s - loss: 0.7269 - categorical_accuracy: 0.7003Epoch 00003: val_categorical_accuracy did not improve\n",
      "1086419/1086419 [==============================] - 167s - loss: 0.7269 - categorical_accuracy: 0.7003 - val_loss: 1.4518 - val_categorical_accuracy: 0.5874\n",
      "Epoch 5/5\n",
      "1085440/1086419 [============================>.] - ETA: 0s - loss: 0.6925 - categorical_accuracy: 0.7098Epoch 00004: val_categorical_accuracy improved from 0.58813 to 0.58963, saving model to dis_model_weights.hdf5\n",
      "1086419/1086419 [==============================] - 155s - loss: 0.6925 - categorical_accuracy: 0.7098 - val_loss: 1.4853 - val_categorical_accuracy: 0.5896\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    # model = keras.models.load_model('current_model.h5')\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    try:\n",
    "        dis_model_1.load_weights(\"dis_model_1_weights.hdf5\")\n",
    "    except IOError as ioe:\n",
    "        print(\"no checkpoints available !\")\n",
    "    dis_model_1.fit(x_train_21_T, x_train_21_C, \n",
    "          validation_data=(x_val_21_T, x_val_21_C),\n",
    "          epochs=5, batch_size=1024, shuffle=True,\n",
    "          callbacks=[tb_callback,checkpointer])\n",
    "    #model.save('current_sent_model.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generative model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Not implemented"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-24T06:58:17.661183Z",
     "start_time": "2017-08-24T06:58:17.655020Z"
    }
   },
   "source": [
    "## model_2: with GRU"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Discriminative model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T19:45:49.466715Z",
     "start_time": "2017-10-27T19:45:48.428575Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "text_seq_input = Input(shape=(MAX_SENT_LEN,), dtype='int32')\n",
    "text_embedding = Embedding(vocab_size, WORD_EMB_SIZE, input_length=MAX_SENT_LEN,\n",
    "                            weights=[trained_embeddings], trainable=True)(text_seq_input)\n",
    "\n",
    "dis_model_2 = Sequential([\n",
    "    Embedding(vocab_size, WORD_EMB_SIZE, weights=[trained_embeddings], \n",
    "               input_length=MAX_SENT_LEN, trainable=True),\n",
    "    GRU(32, return_sequences=True),\n",
    "    Flatten(),\n",
    "    Dense(9, activation='softmax')\n",
    "])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T19:45:49.896605Z",
     "start_time": "2017-10-27T19:45:49.862370Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_2 (Embedding)      (None, 60, 200)           70444000  \n",
      "_________________________________________________________________\n",
      "gru_1 (GRU)                  (None, 60, 32)            22368     \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 1920)              0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 9)                 17289     \n",
      "=================================================================\n",
      "Total params: 70,483,657\n",
      "Trainable params: 70,483,657\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "dis_model_2.compile(loss='categorical_crossentropy', optimizer=Adam(), metrics=['categorical_accuracy'])\n",
    "dis_model_2.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T19:45:50.506445Z",
     "start_time": "2017-10-27T19:45:50.246250Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%rm -rf ./tb_graphs/*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T19:45:50.707496Z",
     "start_time": "2017-10-27T19:45:50.703805Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tb_callback = keras.callbacks.TensorBoard(log_dir='./tb_graphs', histogram_freq=0, write_graph=True, write_images=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T19:45:52.326618Z",
     "start_time": "2017-10-27T19:45:52.321507Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "checkpointer = ModelCheckpoint(filepath=\"dis_model_2_weights.hdf5\", \n",
    "                                    verbose=1,\n",
    "                                    monitor=\"val_categorical_accuracy\",\n",
    "                                    save_best_only=True,\n",
    "                                    mode=\"max\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T19:57:47.071129Z",
     "start_time": "2017-10-27T19:45:52.789618Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no checkpoints available !\n",
      "Train on 1086419 samples, validate on 128341 samples\n",
      "Epoch 1/5\n",
      "1085440/1086419 [============================>.] - ETA: 0s - loss: 1.1280 - categorical_accuracy: 0.5820Epoch 00000: val_categorical_accuracy improved from -inf to 0.57053, saving model to dis_model_2_weights.hdf5\n",
      "1086419/1086419 [==============================] - 105s - loss: 1.1278 - categorical_accuracy: 0.5820 - val_loss: 1.2550 - val_categorical_accuracy: 0.5705\n",
      "Epoch 2/5\n",
      "1085440/1086419 [============================>.] - ETA: 0s - loss: 0.8591 - categorical_accuracy: 0.6644Epoch 00001: val_categorical_accuracy improved from 0.57053 to 0.58595, saving model to dis_model_2_weights.hdf5\n",
      "1086419/1086419 [==============================] - 131s - loss: 0.8591 - categorical_accuracy: 0.6644 - val_loss: 1.2920 - val_categorical_accuracy: 0.5859\n",
      "Epoch 3/5\n",
      "1085440/1086419 [============================>.] - ETA: 0s - loss: 0.7748 - categorical_accuracy: 0.6875Epoch 00002: val_categorical_accuracy improved from 0.58595 to 0.58957, saving model to dis_model_2_weights.hdf5\n",
      "1086419/1086419 [==============================] - 159s - loss: 0.7748 - categorical_accuracy: 0.6876 - val_loss: 1.3501 - val_categorical_accuracy: 0.5896\n",
      "Epoch 4/5\n",
      "1085440/1086419 [============================>.] - ETA: 0s - loss: 0.7247 - categorical_accuracy: 0.7021Epoch 00003: val_categorical_accuracy did not improve\n",
      "1086419/1086419 [==============================] - 157s - loss: 0.7247 - categorical_accuracy: 0.7021 - val_loss: 1.3817 - val_categorical_accuracy: 0.5893\n",
      "Epoch 5/5\n",
      "1085440/1086419 [============================>.] - ETA: 0s - loss: 0.6928 - categorical_accuracy: 0.7114Epoch 00004: val_categorical_accuracy improved from 0.58957 to 0.59470, saving model to dis_model_2_weights.hdf5\n",
      "1086419/1086419 [==============================] - 158s - loss: 0.6928 - categorical_accuracy: 0.7114 - val_loss: 1.4189 - val_categorical_accuracy: 0.5947\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    # model = keras.models.load_model('current_model.h5')\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    try:\n",
    "        dis_model_2.load_weights(\"dis_model_2_weights.hdf5\")\n",
    "    except IOError as ioe:\n",
    "        print(\"no checkpoints available !\")\n",
    "    dis_model_2.fit(x_train_21_T, x_train_21_C, \n",
    "          validation_data=(x_val_21_T, x_val_21_C),\n",
    "          epochs=5, batch_size=1024, shuffle=True,\n",
    "          callbacks=[tb_callback,checkpointer])\n",
    "    #model.save('current_sent_model.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generative model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Not implemented"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {
    "height": "833px",
    "left": "0px",
    "right": "1192px",
    "top": "52px",
    "width": "300px"
   },
   "toc_section_display": "block",
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
